package com.zxk.crawler.task;

import com.zxk.core.common.model.News;
import com.zxk.core.common.util.DateTransUtil;
import com.zxk.crawler.pipeline.NewsPipeline;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Selectable;

import java.util.Date;
import java.util.List;

/**
 * @author zxk
 * @description 职位爬虫任务
 * @date 2020/8/27
 */
@Component
public class NewsProcessor implements PageProcessor {

    @Override
    public void process(Page page) {
        List<Selectable> linkLists = page.getHtml().css("body > div.main.clearfix").links().regex("http://www.ce.cn.*").nodes();

        if(linkLists.size() == 0) {
            //详情页
            saveNews(page);
        } else {
            for (Selectable detailUrl : linkLists) {
                page.addTargetRequest(detailUrl.toString());
            }
        }

    }

    private void saveNews(Page page) {
        News news = new News();
        try {
            news.setAuthor(page.getHtml().css("div#articleText p[style=float:right;]","text").toString());
            String con = page.getHtml().css("div#articleText div.TRS_Editor").toString();
            if(StringUtils.isNotBlank(con)) {
                news.setContent(Jsoup.parse(con).text());
            }
            Date date = DateTransUtil.string2dateZh(page.getHtml().css("span#articleTime", "text").toString());
            news.setCreateTime(new Date());
            news.setDetailLink(page.getUrl().toString());
            news.setNewsTime(date);
            news.setSource(page.getHtml().css("span#articleSource", "text").toString());
            news.setTitle(page.getHtml().css("h1#articleTitle", "text").toString());

            page.putField("news",news);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private Site site = Site.me()
            .setCharset("gbk")
            .setTimeOut(10000)
            .setRetryTimes(3)
            .setRetrySleepTime(3000);
    @Override
    public Site getSite() {
        return site;
    }

    @Autowired
    private NewsPipeline newsPipeline;

    @Scheduled(cron = "0 00 01 ? * *")
//    @Scheduled(fixedDelay = 1000)
    public void jobProcessor() {
        //创建downloader
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
//        httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("58.87.68.189",1080)));
        Spider.create(new NewsProcessor())
                .addUrl("http://www.ce.cn/")
                .addPipeline(this.newsPipeline)
                .setDownloader(httpClientDownloader)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(5)
                .start();
    }
}
